diff --git a/common/spl/Kconfig b/common/spl/Kconfig
index 8021661..a48617d 100644
--- a/common/spl/Kconfig
+++ b/common/spl/Kconfig
@@ -158,7 +158,7 @@
 
 config SPL_LEGACY_IMAGE_SUPPORT
 	bool "Support SPL loading and booting of Legacy images"
-	default y if !TI_SECURE_DEVICE
+	default y if !TI_SECURE_DEVICE && !SPL_LOAD_FIT
 	help
 	  SPL will support loading and booting Legacy images when this option
 	  is y. If this is not set, SPL will move on to other available